#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai

__license__   = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

import re
import time
import unicodedata
from datetime import datetime
from io import StringIO
from operator import attrgetter
from threading import Thread

from calibre.customize.ui import all_metadata_plugins, metadata_plugins
from calibre.ebooks.metadata import authors_to_sort_string, check_issn
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.sources.base import create_log
from calibre.ebooks.metadata.sources.prefs import msprefs
from calibre.ebooks.metadata.xisbn import xisbn
from calibre.utils.date import UNDEFINED_DATE, as_utc, utc_tz
from calibre.utils.formatter import EvalFormatter
from calibre.utils.html2text import html2text
from calibre.utils.icu import lower, primary_sort_key
from polyglot.builtins import as_unicode, iteritems, itervalues
from polyglot.queue import Empty, Queue
from polyglot.urllib import quote, urlparse

# Download worker {{{


class Worker(Thread):

    def __init__(self, plugin, kwargs, abort):
        Thread.__init__(self)
        self.daemon = True

        self.plugin, self.kwargs, self.rq = plugin, kwargs, Queue()
        self.abort = abort
        self.buf = StringIO()
        self.log = create_log(self.buf)

    def run(self):
        start = time.time()
        try:
            self.plugin.identify(self.log, self.rq, self.abort, **self.kwargs)
        except Exception:
            self.log.exception('Plugin', self.plugin.name, 'failed')
        self.plugin.dl_time_spent = time.time() - start

    @property
    def name(self):
        return self.plugin.name


def is_worker_alive(workers):
    for w in workers:
        if w.is_alive():
            return True
    return False

# }}}


# Merge results from different sources {{{

class xISBN(Thread):

    def __init__(self, isbn):
        Thread.__init__(self)
        self.isbn = isbn
        self.isbns = frozenset()
        self.min_year = None
        self.daemon = True
        self.exception = self.tb = None

    def run(self):
        try:
            self.isbns, self.min_year = xisbn.get_isbn_pool(self.isbn)
        except Exception as e:
            import traceback
            self.exception = e
            self.tb = traceback.format_exception()


class ISBNMerge:

    def __init__(self, log):
        self.pools = {}
        self.isbnless_results = []
        self.results = []
        self.log = log
        # The xISBN service has been de-commissioned
        # https://www.oclc.org/developer/news/2018/xid-decommission.en.html
        self.use_xisbn = False

    def isbn_in_pool(self, isbn):
        if isbn:
            for isbns, pool in iteritems(self.pools):
                if isbn in isbns:
                    return pool
        return None

    def pool_has_result_from_same_source(self, pool, result):
        results = pool[1]
        for r in results:
            if r.identify_plugin is result.identify_plugin:
                return True
        return False

    def add_result(self, result):
        isbn = result.isbn
        if isbn:
            pool = self.isbn_in_pool(isbn)
            if pool is None:
                isbns = min_year = None
                if self.use_xisbn:
                    xw = xISBN(isbn)
                    xw.start()
                    xw.join(10)
                    if xw.is_alive():
                        self.log.error('Query to xISBN timed out')
                        self.use_xisbn = False
                    elif xw.exception:
                        self.log.error('Query to xISBN failed:')
                        self.log.debug(xw.tb)
                    else:
                        isbns, min_year = xw.isbns, xw.min_year
                        if not msprefs['find_first_edition_date']:
                            min_year = None
                if not isbns:
                    isbns = frozenset([isbn])
                if isbns in self.pools:
                    # xISBN had a brain fart
                    pool = self.pools[isbns]
                else:
                    self.pools[isbns] = pool = (min_year, [])

            if not self.pool_has_result_from_same_source(pool, result):
                pool[1].append(result)
        else:
            self.isbnless_results.append(result)

    def finalize(self):
        has_isbn_result = False
        for results in itervalues(self.pools):
            if results:
                has_isbn_result = True
                break

        isbn_sources = frozenset()
        if has_isbn_result:
            isbn_sources = self.merge_isbn_results()

        # Now handle results that have no ISBNs
        results = sorted(self.isbnless_results,
                key=attrgetter('relevance_in_source'))
        # Only use results that are from sources that have not also returned a
        # result with an ISBN
        results = [r for r in results if r.identify_plugin not in isbn_sources or not r.identify_plugin.prefer_results_with_isbn]
        if results:
            # Pick only the most relevant result from each source
            seen = set()
            for result in results:
                if msprefs['keep_dups'] or result.identify_plugin not in seen:
                    seen.add(result.identify_plugin)
                    self.results.append(result)
                    result.average_source_relevance = \
                        result.relevance_in_source

        self.merge_metadata_results()

        return self.results

    def merge_metadata_results(self, merge_on_identifiers=False):
        '''
        Merge results with identical title and authors or an identical
        identifier
        '''
        # First title/author
        groups = {}
        for result in self.results:
            title = lower(result.title if result.title else '')
            key = (title, tuple(lower(x) for x in result.authors))
            if key not in groups:
                groups[key] = []
            groups[key].append(result)

        if len(groups) != len(self.results):
            self.results = []
            for rgroup in itervalues(groups):
                rel = [r.average_source_relevance for r in rgroup]
                if len(rgroup) > 1:
                    result = self.merge(rgroup, None, do_asr=False)
                    result.average_source_relevance = sum(rel)/len(rel)
                else:
                    result = rgroup[0]
                self.results.append(result)

        if merge_on_identifiers:
            # Now identifiers
            groups, empty = {}, []
            for result in self.results:
                key = set()
                for typ, val in iteritems(result.identifiers):
                    if typ and val:
                        key.add((typ, val))
                if key:
                    key = frozenset(key)
                    match = None
                    for candidate in list(groups):
                        if candidate.intersection(key):
                            # We have at least one identifier in common
                            match = candidate.union(key)
                            results = groups.pop(candidate)
                            results.append(result)
                            groups[match] = results
                            break
                    if match is None:
                        groups[key] = [result]
                else:
                    empty.append(result)

            if len(groups) != len(self.results):
                self.results = []
                for rgroup in itervalues(groups):
                    rel = [r.average_source_relevance for r in rgroup]
                    if len(rgroup) > 1:
                        result = self.merge(rgroup, None, do_asr=False)
                        result.average_source_relevance = sum(rel)/len(rel)
                    elif rgroup:
                        result = rgroup[0]
                    self.results.append(result)

            if empty:
                self.results.extend(empty)

        self.results.sort(key=attrgetter('average_source_relevance'))

    def merge_isbn_results(self):
        self.results = []
        sources = set()
        for min_year, results in itervalues(self.pools):
            if results:
                for r in results:
                    sources.add(r.identify_plugin)
                self.results.append(self.merge(results, min_year))

        self.results.sort(key=attrgetter('average_source_relevance'))
        return sources

    def length_merge(self, attr, results, null_value=None, shortest=True):
        values = [getattr(x, attr) for x in results if not x.is_null(attr)]
        values = [x for x in values if len(x) > 0]
        if not values:
            return null_value
        values.sort(key=len, reverse=not shortest)
        return values[0]

    def random_merge(self, attr, results, null_value=None):
        values = [getattr(x, attr) for x in results if not x.is_null(attr)]
        return values[0] if values else null_value

    def merge(self, results, min_year, do_asr=True):
        ans = Metadata(_('Unknown'))

        # We assume the shortest title has the least cruft in it
        ans.title = self.length_merge('title', results, null_value=ans.title)

        # No harm in having extra authors, maybe something useful like an
        # editor or translator
        ans.authors = self.length_merge('authors', results,
                null_value=ans.authors, shortest=False)

        # We assume the shortest publisher has the least cruft in it
        ans.publisher = self.length_merge('publisher', results,
                null_value=ans.publisher)

        # We assume the smallest set of tags has the least cruft in it
        ans.tags = self.length_merge('tags', results,
                null_value=ans.tags, shortest=msprefs['fewer_tags'])

        # We assume the longest series has the most info in it
        ans.series = self.length_merge('series', results,
                null_value=ans.series, shortest=False)
        for r in results:
            if r.series and r.series == ans.series:
                ans.series_index = r.series_index
                break

        # Average the rating over all sources
        ratings = []
        for r in results:
            rating = r.rating
            if rating and rating > 0 and rating <= 5:
                ratings.append(rating)
        if ratings:
            ans.rating = round(sum(ratings)/len(ratings))

        # Smallest language is likely to be valid
        ans.language = self.length_merge('language', results,
                null_value=ans.language)

        # Choose longest comments
        ans.comments = self.length_merge('comments', results,
                null_value=ans.comments, shortest=False)

        # Published date
        if min_year:
            for r in results:
                year = getattr(r.pubdate, 'year', None)
                if year == min_year:
                    ans.pubdate = r.pubdate
                    break
            if getattr(ans.pubdate, 'year', None) == min_year:
                min_date = datetime(min_year, ans.pubdate.month, ans.pubdate.day,
                                    tzinfo=utc_tz)
            else:
                min_date = datetime(min_year, 1, 2, tzinfo=utc_tz)
            ans.pubdate = min_date
        else:
            min_date = datetime(3001, 1, 1, tzinfo=utc_tz)
            for r in results:
                if r.pubdate is not None:
                    candidate = as_utc(r.pubdate)
                    min_date = min(min_date, candidate)
            if min_date.year < 3000:
                ans.pubdate = min_date

        # Identifiers
        for r in results:
            ans.identifiers.update(r.identifiers)

        # Cover URL
        ans.has_cached_cover_url = bool([r for r in results if
            getattr(r, 'has_cached_cover_url', False)])

        # Merge any other fields with no special handling (random merge)
        touched_fields = set()
        for r in results:
            if hasattr(r, 'identify_plugin'):
                touched_fields |= r.identify_plugin.touched_fields

        for f in touched_fields:
            if f.startswith('identifier:') or not ans.is_null(f):
                continue
            setattr(ans, f, self.random_merge(f, results,
                null_value=getattr(ans, f)))

        if do_asr:
            avg = [x.relevance_in_source for x in results]
            avg = sum(avg)/len(avg)
            ans.average_source_relevance = avg

        return ans


def merge_identify_results(result_map, log):
    isbn_merge = ISBNMerge(log)
    for plugin, results in iteritems(result_map):
        for result in results:
            isbn_merge.add_result(result)

    return isbn_merge.finalize()

# }}}


def identify(log, abort,  # {{{
        title=None, authors=None, identifiers={}, timeout=30, allowed_plugins=None):
    if title == _('Unknown'):
        title = None
    if authors == [_('Unknown')]:
        authors = None
    start_time = time.time()

    plugins = [p for p in metadata_plugins(['identify'])
        if p.is_configured() and (allowed_plugins is None or p.name in allowed_plugins)]

    kwargs = {
        'title': title,
        'authors': authors,
        'identifiers': identifiers,
        'timeout': timeout,
    }

    log('Running identify query with parameters:')
    log(kwargs)
    log('Using plugins:', ', '.join(['%s %s' % (p.name, p.version) for p in plugins]))
    log('The log from individual plugins is below')

    workers = [Worker(p, kwargs, abort) for p in plugins]
    for w in workers:
        w.start()

    first_result_at = None
    results = {}
    for p in plugins:
        results[p] = []
    logs = {w.plugin: w.buf for w in workers}

    def get_results():
        found = False
        for w in workers:
            try:
                result = w.rq.get_nowait()
            except Empty:
                pass
            else:
                results[w.plugin].append(result)
                found = True
        return found

    wait_time = msprefs['wait_after_first_identify_result']
    while True:
        time.sleep(0.2)

        if get_results() and first_result_at is None:
            first_result_at = time.time()

        if not is_worker_alive(workers):
            break

        if (first_result_at is not None and time.time() - first_result_at > wait_time):
            log.warn('Not waiting any longer for more results. Still running'
                    ' sources:')
            for worker in workers:
                if worker.is_alive():
                    log.debug('\t' + worker.name)
            abort.set()
            break

    while not abort.is_set() and get_results():
        pass

    sort_kwargs = dict(kwargs)
    for k in list(sort_kwargs):
        if k not in ('title', 'authors', 'identifiers'):
            sort_kwargs.pop(k)

    longest, lp = -1, ''
    for plugin, presults in iteritems(results):
        presults.sort(key=plugin.identify_results_keygen(**sort_kwargs))

        # Throw away lower priority results from the same source that have exactly the same
        # title and authors as a higher priority result
        filter_results = set()
        filtered_results = []
        for r in presults:
            key = (r.title, tuple(r.authors))
            if key not in filter_results:
                filtered_results.append(r)
                filter_results.add(key)
        results[plugin] = presults = filtered_results

        plog = logs[plugin].getvalue().strip()
        log('\n'+'*'*30, plugin.name, '%s' % (plugin.version,), '*'*30)
        log('Found %d results'%len(presults))
        time_spent = getattr(plugin, 'dl_time_spent', None)
        if time_spent is None:
            log('Downloading was aborted')
            longest, lp = -1, plugin.name
        else:
            log('Downloading from', plugin.name, 'took', time_spent)
            if time_spent > longest:
                longest, lp = time_spent, plugin.name
        for r in presults:
            log('\n\n---')
            try:
                log(str(r))
            except TypeError:
                log(repr(r))
        if plog:
            log(plog)
        log('\n'+'*'*80)

        dummy = Metadata(_('Unknown'))
        for i, result in enumerate(presults):
            for f in plugin.prefs['ignore_fields']:
                if ':' not in f:
                    setattr(result, f, getattr(dummy, f))
                if f == 'series':
                    result.series_index = dummy.series_index
            result.relevance_in_source = i
            result.has_cached_cover_url = (
                plugin.cached_cover_url_is_reliable and plugin.get_cached_cover_url(result.identifiers) is not None)
            result.identify_plugin = plugin
            if msprefs['txt_comments']:
                if plugin.has_html_comments and result.comments:
                    result.comments = html2text(result.comments)

    log('The identify phase took %.2f seconds'%(time.time() - start_time))
    log('The longest time (%f) was taken by:'%longest, lp)
    log('Merging results from different sources')
    start_time = time.time()
    results = merge_identify_results(results, log)

    log('We have %d merged results, merging took: %.2f seconds' %
            (len(results), time.time() - start_time))
    tm_rules = msprefs['tag_map_rules']
    pm_rules = msprefs['publisher_map_rules']
    s_rules = msprefs['series_map_rules']
    if tm_rules or pm_rules or s_rules:
        from calibre.ebooks.metadata.tag_mapper import map_tags
    am_rules = msprefs['author_map_rules']
    if am_rules:
        from calibre.ebooks.metadata.author_mapper import compile_rules, map_authors
        am_rules = compile_rules(am_rules)

    # normalize unicode strings
    def n(x):
        return unicodedata.normalize('NFC', as_unicode(x or '', errors='replace'))
    for r in results:
        if r.tags:
            r.tags = list(map(n, r.tags))
        if r.authors:
            r.authors = list(map(n, r.authors))
        if r.author_sort:
            r.author_sort = n(r.author_sort)
        if r.title:
            r.title = n(r.title)
        if r.publisher:
            r.publisher = n(r.publisher)
        if r.comments:
            r.comments = n(r.comments)
    max_tags = msprefs['max_tags']
    for r in results:
        if tm_rules:
            r.tags = map_tags(r.tags, tm_rules)
        r.tags = r.tags[:max_tags]
        if getattr(r.pubdate, 'year', 2000) <= UNDEFINED_DATE.year:
            r.pubdate = None
        if pm_rules and r.publisher:
            pubs = map_tags([r.publisher], pm_rules, separator='')
            r.publisher = pubs[0] if pubs else ''
        if s_rules and r.series:
            ss = map_tags([r.series], s_rules, separator='')
            r.series = ss[0] if ss else ''

    if msprefs['swap_author_names']:
        for r in results:
            def swap_to_ln_fn(a):
                if ',' in a:
                    return a
                parts = a.split(None)
                if len(parts) <= 1:
                    return a
                surname = parts[-1]
                return '%s, %s' % (surname, ' '.join(parts[:-1]))
            r.authors = [swap_to_ln_fn(a) for a in r.authors]

    if am_rules:
        for r in results:
            new_authors = map_authors(r.authors, am_rules)
            if new_authors != r.authors:
                r.authors = new_authors
                r.author_sort = authors_to_sort_string(r.authors)

    return results
# }}}


def urls_from_identifiers(identifiers, sort_results=False):  # {{{
    identifiers = {k.lower():v for k, v in iteritems(identifiers)}
    ans = []
    keys_left = set(identifiers)

    def add(name, k, val, url):
        ans.append((name, k, val, url))
        keys_left.discard(k)

    rules = msprefs['id_link_rules']
    if rules:
        formatter = EvalFormatter()
        for k, val in iteritems(identifiers):
            val = val.replace('|', ',')
            vals = {
                'id':str(quote(val if isinstance(val, bytes) else val.encode('utf-8'))),
                'id_unquoted': str(val),
            }
            items = rules.get(k) or ()
            for name, template in items:
                try:
                    url = formatter.safe_format(template, vals, '', vals)
                except Exception:
                    import traceback
                    traceback.format_exc()
                    continue
                add(name, k, val, url)
    for plugin in all_metadata_plugins():
        try:
            for id_type, id_val, url in plugin.get_book_urls(identifiers):
                add(plugin.get_book_url_name(id_type, id_val, url), id_type, id_val, url)
        except Exception:
            pass
    isbn = identifiers.get('isbn', None)
    if isbn:
        add(isbn, 'isbn', isbn,
            'https://www.worldcat.org/isbn/'+isbn)
    doi = identifiers.get('doi', None)
    if doi:
        add('DOI', 'doi', doi,
            'https://dx.doi.org/'+doi)
    arxiv = identifiers.get('arxiv', None)
    if arxiv:
        add('arXiv', 'arxiv', arxiv,
            'https://arxiv.org/abs/'+arxiv)
    oclc = identifiers.get('oclc', None)
    if oclc:
        add('OCLC', 'oclc', oclc,
            'https://www.worldcat.org/oclc/'+oclc)
    issn = check_issn(identifiers.get('issn', None))
    if issn:
        add(issn, 'issn', issn,
            'https://www.worldcat.org/issn/'+issn)
    q = {'http', 'https', 'file'}
    for k, url in iteritems(identifiers):
        if url and re.match(r'ur[il]\d*$', k) is not None:
            url = url[:8].replace('|', ':') + url[8:].replace('|', ',')
            if url.partition(':')[0].lower() in q:
                parts = urlparse(url)
                name = parts.netloc or parts.path
                add(name, k, url, url)
    for k in tuple(keys_left):
        val = identifiers.get(k)
        if val:
            url = val[:8].replace('|', ':') + val[8:].replace('|', ',')
            if url.partition(':')[0].lower() in q:
                parts = urlparse(url)
                name = parts.netloc or parts.path
                add(name, k, url, url)
    if sort_results:

        def url_key(x):
            return primary_sort_key(str(x[0]))

        ans = sorted(ans, key=url_key)
    return ans
# }}}


def tests(start=0, limit=256):  # tests {{{
    # To run these test use: calibre-debug -c "from calibre.ebooks.metadata.sources.identify import tests; tests()"
    from calibre.ebooks.metadata.sources.test import authors_test, test_identify, title_test
    tests = [
            (
                {'title':'Magykal Papers',
                    'authors':['Sage']},
                [title_test('Septimus Heap: The Magykal Papers', exact=True)],
            ),


            (  # An e-book ISBN not on Amazon, one of the authors is unknown to Amazon
                {'identifiers':{'isbn': '9780307459671'},
                    'title':'Invisible Gorilla', 'authors':['Christopher Chabris']},
                [title_test('The Invisible Gorilla: And Other Ways Our Intuitions Deceive Us', exact=True)]

            ),

            (  # Test absence of identifiers
                {'title':'Learning Python',
                    'authors':['Lutz']},
                [title_test('Learning Python',
                    exact=True), authors_test(['Mark J. Lutz', 'David Ascher'])
                 ]

            ),

            (  # Sophisticated comment formatting
                {'identifiers':{'isbn': '9781416580829'}},
                [title_test('Angels & Demons',
                    exact=True), authors_test(['Dan Brown'])]
            ),

            (  # A newer book
                {'identifiers':{'isbn': '9780316044981'}},
                [title_test('The Heroes', exact=True),
                    authors_test(['Joe Abercrombie'])]

            ),

        ]
    # test_identify(tests[1:2])
    test_identify(tests[start:limit])
# }}}
